package com.arnold.test.transformations

import org.apache.spark.{SparkConf, SparkContext}

/**
  * Created by arnold.zhu on 2017/7/17.
  */
object Distinct {

  def main(args: Array[String]): Unit = {
    val conf = new SparkConf().setMaster("local").setAppName("Spark01Learn")
    val sc = new SparkContext(conf)

    val kv2 = sc.parallelize(List(("A", 3), ("A", 4), ("C", 3), ("A", 5), ("B", 5)))
    // 过滤重复的
    val arr1 = kv2.distinct.collect
    for (a <- arr1) {
      println(a)
    }

  }

}
